/*
代码需求：将一个目录中所有的文件合并称为一个文件，原文件在目标文件表现就是一行，并在目标文件中每一行添加索引。如:1001
这里给出一个例子:
如原文件：
hello scala
hello python
hello java
目标文件：
1001 hello scala hello python hello java
*/
import java.io.PrintWriter
import java.io.File
import scala.reflect.io.Directory

object directoryTraversal {
    def main(args: Array[String]) {
        // for(d <- subDir(new File("/data/nlp_3qi/input_tfidf_dir")))
        // println(d)
        val filePath = "/tmp/merge.txt"
        writeContextToFile(filePath)
    }
    def subDir(dir:File):Iterator[File] = {
        //val children = dir.listFiles().filter(_.isDirectory())
        val dirs = dir.listFiles().filter(_.isDirectory())
        val files = dir.listFiles().filter(_.isFile())
        // children.toIterator ++ children.toIterator.flatMap(subDir _)
        files.toIterator ++ dirs.toIterator.flatMap(subDir _)
    }
    def getContextFromFile(file:String) :String = {
        import scala.io.Source
        val fileContext = Source.fromFile(file).mkString
        import scala.util.matching.Regex
        val line = fileContext.replaceAll("[^\u4e00-\u9fa5]+"," ")
        return line
    }
    def writeContextToFile(file:String) = {
        val write = new PrintWriter(new File(file))
        var index = 1001
        val fileDirPath="/data/nlp_3qi/input_tfidf_dir"
        for(fd <- subDir(new File(fileDirPath))) {
            val line = getContextFromFile(fd.toString)
            val lineJoin = index.toString + "\t" + line
            write.write(lineJoin)
            write.write("\n")
            index += 1
        }
        write.close()
    }
}